#------------------------------------------------------------------------------
# this code transform large file on Carbonate cluster
#==============================================================================

load_library = c('bit64','data.table','fst','future.apply','stringr','logger','vroom')
invisible(lapply(load_library, function(x) library(x, character.only=TRUE, quietly= TRUE)))

bucket = file.path('/N','project','iuni_doctorshopping')

args=commandArgs(TRUE)

in_file = args[[1]]
out_file = args[[2]]

if (!file.exists(out_file)) {
	logger::log_info("now reading data : ", in_file)	

	data = vroom(in_file,
			delim='|',
			col_names=TRUE,
			num_threads = parallel::detectCores(),
			locale=locale(tz="America/Indiana/Indianapolis"))

	write_fst(data, out_file, 100)

}

